#coding:utf-8
'''
Created on 2015年4月22日

@author: leoZ
'''
from time import sleep
from com.cyou.dataclean.LogParser import LogParser
import logging
from com.cyou.repository.EtlJob import EtlJobDao
from com.cyou.repository.EtlJobSource import EtlJobSourceDao
import os
from datetime import date
from com.cyou.repository.EtlFtpSource import EtlFtpSourceDao
from com.cyou.com.StringUtils import repalcePathStr

logging.basicConfig()
log = logging.getLogger("FtpReceive")
log.setLevel(logging.DEBUG)
#数据清洗的控制类
class DataClean(object):
    #查看任务列表。检查那些文件 db
    def getDataCleanTaskList(self):
        log.debug("任务列表-开始")
        self.etlJobDao = EtlJobDao()
        self.etlJobSourceDao = EtlJobSourceDao()
        jobs = self.etlJobDao.loadAllBySystemName("DC")
        jobSystemDict = {}
        for job in jobs:
            #job装态必须是Ready的才处理
            if job.lastJobStatus != 'Ready':
                log.debug("任务状态不是Ready不进行清洗:任务名:%s-任务状态:%s" % (job.etlJobName, job.lastJobStatus))
                continue
            #获取Soure
            jobSource = self.etlJobSourceDao.loadAllJobSourceByJobName("HQL", job.etlJobName)
            if len(jobSource) != 1:
                log.debug("获取jobSource有误")
                raise "Error"
            source = jobSource[0].source
            #添加job到待处理对列中
            if not jobSystemDict.has_key(source):
                jobSystemDict[source] = []
            jobSystemDict[source].append(job.etlJobName)
            log.debug(("任务清单：系统名:%s-任务名:%s")% (source, job.etlJobName))
        self.jobSystemDict = jobSystemDict
        log.debug("任务列表-完毕")
        pass
    #返回system的路径
    def getSysPath(self, systemName):
        basePath = self.basePath
        return ("%s\\%s\\%s\\%s.ready") % (basePath,str(self.curTime), systemName, systemName)
    #根椐任务列表，检看就绪文件
    def checkReadyFile(self):
        log.debug("检看就绪文件-开始")
        jobSystemDict = self.jobSystemDict
        cleanDict = {}
        allCnt = len(jobSystemDict)
        curCnt = 0
        log.debug("需要检验的系统共有:" + str(allCnt))
        for key in jobSystemDict.keys():
            curCnt = curCnt + 1
            log.debug("开始检验系统-%s-系统名:%s" % (str(curCnt), key))
            sysPath = self.getSysPath(key)
            if os.path.exists(sysPath):
                log.debug("系统%s就绪" % key)
                cleanDict[key] = jobSystemDict[key]
            else:
                log.debug("系统%s没有就绪" % key)
        self.cleanDict = cleanDict
        log.debug(cleanDict)
        log.debug("检看就绪文件-完毕")
    #1.读取xml
    #2.清洗文件
    #3.生成完成文件、错误文件error、无用文件bypass
    def parseFile(self,logPath, xmlConfigPath, date):
        log.debug(logPath + xmlConfigPath + date)
        logParser = LogParser()
        return logParser.parseLog(logPath, xmlConfigPath, date)
    #获取日志路径
    def getCleanFilePath(self, jobName):
        #"E:\data\receive\2015-04-21\CY_UC\stderr.log.2015-04-21"
        jobSource = self.etlJobSourceDao.loadAllJobSourceByJobName("HQL", jobName)
        if len(jobSource) != 1:
            log.debug("获取jobSource有误")
            raise "Error"
        source = jobSource[0].source
        self.etlFtpSourceDao = EtlFtpSourceDao()
        ftpSource = self.etlFtpSourceDao.loadAllJobSourceByJobName("HQL", jobName)
        if len(ftpSource) == 1:
            ftpFilePath = ftpSource[0].ftpFilePath
            fileName = ftpFilePath[ftpFilePath.rindex("/")+1:]
            fileName = repalcePathStr(fileName, str(self.curTime))
            return "%s\\%s\\%s\\%s" % (self.basePath, str(self.curTime), source, fileName)
        return None
    #获取xml配置名
    def getXmlConfigPath(self, jobName):
        jobSource = self.etlJobSourceDao.loadAllJobSourceByJobName("HQL", jobName)
        if len(jobSource) != 1:
            log.debug("获取jobSource有误")
            raise "Error"
        return jobSource[0].convertFileHeader
    #清洗已经就绪的文件
    def parseFiles(self):
        log.debug("清洗已经就绪的文件-开始")
        self.finishJob = []
        for key in self.cleanDict:
            log.debug("系统%s开始清洗" % key)
            for jobName in self.cleanDict[key]:
                log.debug("任务%s开始清洗" % jobName)              
                logPath = self.getCleanFilePath(jobName)
                xmlConfigPath = self.getXmlConfigPath(jobName)
                date = str(self.curTime)
                if self.parseFile(logPath, xmlConfigPath, date):
                    self.finishJob.append(jobName)
        log.debug("清洗已经就绪的文件-完毕")
    #设置完成任务的状态
    def updateFinishJob(self):
        finCnt = len(self.finishJob)
        if finCnt <= 0:
            return
        for jobName in self.finishJob:
            self.etlJobDao.updateJobJobStatus("Done", "DC", jobName)
        log.debug("本次任务完成%s个任务" % (str(finCnt)))
    #主逻辑。用于完整调度
    def process(self):
        log.debug("清洗逻辑处理-开始") 
        #查看任务列表。检查那些文件 db
        self.getDataCleanTaskList()
        #根椐任务列表，检看就绪文件
        self.checkReadyFile()
        #清洗已经就绪的文件
        self.parseFiles()
        #如果清洗成功。修改任务况态为done。
        self.updateFinishJob()
        log.debug("清洗逻辑处理-完毕")
    def init(self):
        self.basePath =  "E:\\data\\receive"
        self.curTime = date.today() 
        self.finishJob = None
    def start(self):
        while True:
            self.process()
            sleep(5)
if __name__ == '__main__':
    dataClean = DataClean()
    dataClean.init()
    dataClean.start()
    pass